package com.robet.webmagic.processor;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

public class OschinaRepoPageProcessor implements PageProcessor{
	private Site site = Site.me().setRetryTimes(3).setSleepTime(100);

    @Override
    public void process(Page page) {
        page.addTargetRequests(page.getHtml().links().regex("(https\\://my\\.oschina\\.net/\\w+/\\w+)").all());
//        page.putField("author", page.getUrl().regex("https\\://my\\.oschina\\.net/(\\w+)/.*").toString());
//        page.putField("name", page.getHtml().xpath("//h1[@class='news-list-container']/strong/a/text()").toString());
//        if (page.getResultItems().get("name")==null){
//            //skip this page
//            page.setSkip(true);
//        }
        page.putField("readme", page.getHtml().xpath("//div[@class='news-list-container']").xpath("//h3").links());
     // 部分三：从页面发现后续的url地址来抓取
//        page.addTargetRequests(page.getHtml().links().regex("(https://github\\.com/[\\w\\-]+/[\\w\\-]+)").all());
    }

    @Override
    public Site getSite() {
        return site;
    }


}
